package webmagic;

import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import utils.HttpHelper;

import java.util.HashSet;
import java.util.Set;

/**
 * Created by weishaofei on 2018/11/27.
 */
@Slf4j
public class JDPawler implements PageProcessor {
    private static Set<String> PRODUCTS = new HashSet<String>();

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
    private static final String URL_PREFIX = "http://p.3.cn/prices/mgets?skuIds=J_";

    public static void main(String[] args) {
        Spider.create(new JDPawler()).addUrl("https://item.jd.com/4749506.html").thread(5).run();
    }

    public void process(Page page) {
        Html html = page.getHtml();
        page.addTargetRequests(html.links().regex("https://item.jd.com/.*").all());
        page.addTargetRequests(html.links().regex("https://list.jd.com/.*").all());
        String url = page.getUrl().toString();
        if (!url.contains("item")) {
            return;
        }
        String id = url.substring(20, url.lastIndexOf(".html"));
        String firstCategory = html.css("div.crumb-wrap").css("div.item").css("div.first").xpath("//a/text()").toString().trim();
        String secondCategory = html.css("div.crumb-wrap").css("div.item").xpath("//a[@clstag='shangpin|keycount|product|mbNav-2']/text()").toString();
        String thridCategory = html.css("div.crumb-wrap").css("div.item").xpath("//a[@clstag='shangpin|keycount|product|mbNav-3']/text()").toString();
        String brand = html.css("div.crumb-wrap").css("div.item").xpath("//a[@clstag='shangpin|keycount|product|mbNav-5']/text()").toString();
        String name = html.css("div.sku-name").xpath("//div/text()").toString().trim();
        String price = "";
        try {
            String content = HttpHelper.get(URL_PREFIX + id).getContent().replace("[", "").replace("]", "");
            PriceRet ret = JSONObject.parseObject(content, PriceRet.class);
            price = ret.getP();
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println(firstCategory + ", " + secondCategory + ", " + thridCategory + ", " + name + "," + id + ", " + price);
    }

    public Site getSite() {
        return site;
    }


    @Data
    private static final class PriceRet {
        private String op;
        private String m;
        private String id;
        private String p;
    }
}